package com.nlp.mallet;

import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.regex.Pattern;

import cc.mallet.pipe.CharSequence2TokenSequence;
import cc.mallet.pipe.FeatureSequence2FeatureVector;
import cc.mallet.pipe.Input2CharSequence;
import cc.mallet.pipe.Pipe;
import cc.mallet.pipe.PrintInputAndTarget;
import cc.mallet.pipe.SerialPipes;
import cc.mallet.pipe.Target2Label;
import cc.mallet.pipe.TokenSequence2FeatureSequence;
import cc.mallet.pipe.TokenSequenceLowercase;
import cc.mallet.pipe.TokenSequenceRemoveStopwords;
import cc.mallet.pipe.iterator.FileIterator;
import cc.mallet.types.InstanceList;

public class ImportData {
	
	Pipe pipe;
	public ImportData() {
		pipe = buildPipe();
	}
	private Pipe buildPipe() {
		ArrayList<Pipe> pipeList = new ArrayList<Pipe>();
		//从文件对象读取数据
		pipeList.add(new Input2CharSequence("UTF-8"));
		//用于构成令牌的正则表达式。
		 //    "\\S+"   (anything not whitespace)
        //    "\\w+"    ( A-Z, a-z, 0-9, _ )  
        //    "[\\p{L}\\p{N}_]+|[\\p{P}]+"   一组只有字母和数字或一组只有标点符号的组
		Pattern tokenPattern = Pattern.compile("[\\\\p{L}\\\\p{N}_]+|[\\\\p{P}]+");
		pipeList.add(new CharSequence2TokenSequence(tokenPattern));
		//所有的token转小写
		pipeList.add(new TokenSequenceLowercase());
		//去除停用词的token
		pipeList.add(new TokenSequenceRemoveStopwords(false,false));
		
		pipeList.add(new TokenSequence2FeatureSequence());
		pipeList.add(new Target2Label());
		pipeList.add(new FeatureSequence2FeatureVector());
		pipeList.add(new PrintInputAndTarget());
		return new SerialPipes(pipeList);
	}
	
	/**
	 * 从文件夹读取数据
	 * @param directory
	 * @return
	 */
	public InstanceList readDirectory(File directory) {
		return readDirectory(new File[] {directory});
	}
	
	private InstanceList readDirectory(File[] directories) {
		//http://hao.jobbole.com/mallet/
		FileIterator iterator =
	            new FileIterator(directories,
	                             new TxtFilter(),
	                             FileIterator.LAST_DIRECTORY);
		
		
		InstanceList instances = new InstanceList(pipe);
		instances.addThruPipe(iterator);
		//instances.addThruPipe();
		return instances;
	}
	
	
	public InstanceList readDirectory(String dic) {
		FileIterator iterator =
	            new FileIterator(dic,
	                             new TxtFilter());
		
		
		InstanceList instances = new InstanceList(pipe);
		instances.addThruPipe(iterator);
		//instances.addThruPipe();
		return instances;
	}

}
